/*
 * Comp 479, Fall 2010 Project 1
 *
 * Authors:
 * Boulat Oulmachev #5734266
 * Ivan Khrisanov #9064591
 * Sergiy Samus #4433556
 *
 * Spimi.java
 * Static class that performs SPIMI algorithm
 */

package domain;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.util.HashMap;
import java.util.Map;
import java.util.StringTokenizer;

import model.newmodel.Index;
import model.newmodel.InvalidTermException;
import model.newmodel.Term;
import util.collectionStats.CollectionStatsRegistry;
import util.collectionStats.CollectionStatsRegistry.Stats;

/**
 * @author Ivan Khrisanov, Segriy Samus, Boulat Oulmachev ; Latest reversions:
 *         Fabrice Bloncourt, Paul Smelser and Simon Symeonidis
 * @since December 1, 2010
 * @version Modified to read HTML instead of the RUETERS files. Iterates through
 *          directories and retrieves all files ending in html or php
 */
public class Spimi {

	private static int docCount = 0;
	private static int chkSgmFiles = 0;

	/*
	 * Structure to hold temporary blocks (dictionaries) before they are written
	 * on the disk
	 */
	public static int execute(String rootPath) throws Exception {

		/*
		 * Getting all the files from the directory
		 */
		if (!(new File(rootPath)).exists()) {
			System.out.println("Wrong path: " + rootPath
					+ "\nNothing to index\nPlease check your path");
			System.exit(0);
		}

		String[] files = new File(rootPath).list();

		for (String file : files) {
			if ((new File(rootPath + "/" + file)).isDirectory()) {
				Spimi.execute(rootPath + "/" + file);
			}
		}

		int docsCounter = 0;
		int wordsCounter = 0;
		Map<Integer, Integer> docsLengths = new HashMap<Integer, Integer>();

		/*
		 * Selecting sgm files
		 */
		for (String file : files) {
			if (file.matches("([^\\s]+(\\.(?i)(html|php|txt))$)")) {
				chkSgmFiles++;
				Index index = new Index();
				long startTime = System.currentTimeMillis();

				System.out.print("Indexing: " + file);

				/*
				 * Opening an sgm file
				 */
				BufferedReader reader = new BufferedReader(new FileReader(
						new File(rootPath + "/" + file)));

				String textLine;
				int docId = docCount++;
				System.out.println(docId);
				int docLength = 0; // in words

				/*
				 * Reading source file line-by-line
				 */
				while ((textLine = reader.readLine()) != null) {

					textLine = textLine.replaceAll("<[^>]*>|&lt;[^>]*>", " ");
					textLine = textLine.replaceAll("[\\p{Punct}]", " ");
					textLine = textLine.replaceAll("[\\d]", " ");

					StringTokenizer st = new StringTokenizer(textLine);
					while (st.hasMoreTokens()) {
						try {
							Term term = Term.newInstance(st.nextToken());
							index.addRecord(term, docId);
							docLength++;
							wordsCounter++;
						} catch (InvalidTermException ignore) {
						}
					}
				}

				/*
				 * Saving index (block) on the disk
				 */
				index.saveTo("/Users/paulsmelser/Documents/School/comp479/completeFinalProject/finalVersion/results/tmp-index/"
						+ Integer.toString(chkSgmFiles + 1000000).substring(1)
						+ ".dict");

				/*
				 * Measuring execution time
				 */
				System.out.println(" ("
						+ (System.currentTimeMillis() - startTime) + " ms.)");
			}
		}

		CollectionStatsRegistry statsRegistry = new CollectionStatsRegistry();
		statsRegistry.setStat(Stats.TOTAL_DOCS, docsCounter);
		statsRegistry.setStat(Stats.AVG_DOC_LENGTH, (double) wordsCounter
				/ docsCounter);
		statsRegistry.setStat(Stats.DOCS_LENGHTS, docsLengths);
		statsRegistry.saveTo(rootPath + "stats.registry");

		if (chkSgmFiles == 0) {
			System.out.println("No sgm files found on the path: " + rootPath
					+ "\nNothing to index\nPlease check your path");
			System.exit(0);
		}
		return docCount;
	}
}
